package com.catmiao.spark.rdd.operator.transform

import org.apache.spark.{SparkConf, SparkContext}

/**
 * @title: RDD_Operator_01_transform
 * @projectName spark_study
 * @description: TODO
 * @author ChengMiao
 * @date 2024/2/2 11:48
 */
object RDD_Operator_09_transform {

  def main(args: Array[String]): Unit = {
    val sparkCon = new SparkConf().setMaster("local[*]").setAppName("rdd")

    val sparkContext = new SparkContext(sparkCon)


    val rdd = sparkContext.makeRDD(List(1,2,3,4,5,6),3)

    // coalesce 缩减分区
    /**
     * coalesce 默认情况下不会将同一个分区的数据打乱。这种情况下可能会导致数据倾斜
     *          param2: 是否打乱分区 shuffle
     */
    val rdd2 = rdd.coalesce(2,true)

    rdd2.saveAsTextFile("output")





    sparkContext.stop()
  }

}
